{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For Colab only!\n",
    "\n",
    "try:\n",
    "  # %tensorflow_version only exists in Colab.\n",
    "  %tensorflow_version 2.x\n",
    "except Exception:\n",
    "  pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "cell_style": "split"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "cell_style": "split"
   },
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "cell_style": "split",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.1.0\n",
      "WARNING:tensorflow:From <ipython-input-3-2e82a26757ae>:2: is_gpu_available (from tensorflow.python.framework.test_util) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `tf.config.list_physical_devices('GPU')` instead.\n",
      "True\n"
     ]
    }
   ],
   "source": [
    "print(tf.__version__)\n",
    "print(tf.test.is_gpu_available())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "cell_style": "split"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.4.0\n",
      "True\n"
     ]
    }
   ],
   "source": [
    "print(torch.__version__)\n",
    "print(torch.cuda.is_available())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Clip / Clamp\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "cell_style": "split"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[[-7.1896863  -9.555027    0.04510403]\n",
      " [-7.7503204   0.2455616   8.42992   ]], shape=(2, 3), dtype=float32)\n",
      "tf.Tensor(8.42992, shape=(), dtype=float32)\n",
      "tf.Tensor(-9.555027, shape=(), dtype=float32)\n",
      "tf.Tensor(\n",
      "[[2. 2. 2.]\n",
      " [2. 2. 5.]], shape=(2, 3), dtype=float32)\n",
      "tf.Tensor(\n",
      "[[ True  True  True]\n",
      " [ True  True  True]], shape=(2, 3), dtype=bool)\n",
      "tf.Tensor(16.558487, shape=(), dtype=float32)\n",
      "tf.Tensor(\n",
      "[[-2.1709974  -2.8852355   0.01361961]\n",
      " [-2.3402865   0.07414977  2.5454984 ]], shape=(2, 3), dtype=float32)\n",
      "tf.Tensor(5.0, shape=(), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "grad = tf.random.uniform([2,3], \n",
    "                         minval=-10, \n",
    "                         maxval=10)\n",
    "\n",
    "print(grad)\n",
    "\n",
    "print(tf.reduce_max(grad))\n",
    "print(tf.reduce_min(grad))\n",
    "\n",
    "print(tf.clip_by_value(grad, 2,5))\n",
    "\n",
    "# tf.minimum(tf.maximum(x,min),max)== tf.clip_by_value(x, min, max)\n",
    "# tf.nn.relu(x) == tf.maxium(x,0)\n",
    "\n",
    "print(tf.nn.relu(grad) == tf.maximum(grad, 0))\n",
    "\n",
    "print(tf.norm(grad))\n",
    "grad_cliped = tf.clip_by_norm(grad, 5)\n",
    "print(grad_cliped)\n",
    "print(tf.norm(grad_cliped))\n",
    "\n",
    "torch.nn.utils.clip_grad_norm()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "cell_style": "split"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 9.4082, -9.7744, -9.1278],\n",
      "        [-4.1731, -7.8850,  7.2041]])\n",
      "tensor(9.4082)\n",
      "tensor(-9.7744)\n",
      "tensor([[5., 2., 2.],\n",
      "        [2., 2., 5.]])\n",
      "tensor([[9.4082, 5.0000, 5.0000],\n",
      "        [5.0000, 5.0000, 7.2041]])\n",
      "tensor([[True, True, True],\n",
      "        [True, True, True]])\n",
      "tensor(19.9714)\n",
      "tensor([[ 9.4082, -9.7744, -9.1278],\n",
      "        [-4.1731, -7.8850,  7.2041]])\n"
     ]
    }
   ],
   "source": [
    "grad = torch.rand(2,3)*20 - 10\n",
    "\n",
    "print(grad)\n",
    "\n",
    "print(grad.max())\n",
    "print(grad.min())\n",
    "\n",
    "print(torch.clamp(grad, min=2, max=5))\n",
    "print(grad.clamp(5,10))\n",
    "\n",
    "# torch.clamp(x,min=0)== relu(x)\n",
    "# print(torch.functional.F.relu(grad))\n",
    "# print(torch.clamp(grad, min=0))\n",
    "print(torch.clamp(grad, min=0)\n",
    "      ==torch.functional.F.relu(grad))\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Where\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "cell_style": "split"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor(\n",
      "[[0.18631041 0.7320858  0.55777454]\n",
      " [0.9596844  0.39142764 0.588768  ]\n",
      " [0.5940238  0.878965   0.5175885 ]], shape=(3, 3), dtype=float32)\n",
      "tf.Tensor(\n",
      "[[False  True  True]\n",
      " [ True False  True]\n",
      " [ True  True  True]], shape=(3, 3), dtype=bool)\n",
      "tf.Tensor(\n",
      "[[0 1]\n",
      " [0 2]\n",
      " [1 0]\n",
      " [1 2]\n",
      " [2 0]\n",
      " [2 1]\n",
      " [2 2]], shape=(7, 2), dtype=int64)\n",
      "tf.Tensor(\n",
      "[[0. 1. 1.]\n",
      " [1. 0. 1.]\n",
      " [1. 1. 1.]], shape=(3, 3), dtype=float32)\n",
      "tf.Tensor(\n",
      "[0.7320858  0.55777454 0.9596844  0.588768   0.5940238  0.878965\n",
      " 0.5175885 ], shape=(7,), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "a = tf.random.uniform([3,3])\n",
    "mask = a > 0.5\n",
    "i = tf.ones([3,3])\n",
    "o = tf.zeros([3,3])\n",
    "\n",
    "print(a)\n",
    "print(mask)\n",
    "print(tf.where(mask))\n",
    "print(tf.where(mask, i,o))\n",
    "\n",
    "indices = tf.where(mask)\n",
    "print(tf.gather_nd(a, indices))\n",
    "\n",
    "tf.gather()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "cell_style": "split"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.3384, 0.0522, 0.2639],\n",
      "        [0.0742, 0.8032, 0.6902],\n",
      "        [0.8284, 0.4650, 0.0256]])\n",
      "tensor([[False, False, False],\n",
      "        [False,  True,  True],\n",
      "        [ True, False, False]])\n",
      "tensor([[0., 0., 0.],\n",
      "        [0., 1., 1.],\n",
      "        [1., 0., 0.]])\n"
     ]
    }
   ],
   "source": [
    "a = torch.rand(3,3)\n",
    "mask = a > 0.5\n",
    "i = torch.ones(3,3)\n",
    "o = torch.zeros(3,3)\n",
    "\n",
    "print(a)\n",
    "print(mask)\n",
    "print(torch.where(mask, i,o))\n",
    "\n",
    "torch.gather()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Others\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "cell_style": "split"
   },
   "source": [
    "scatter_nd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "cell_style": "split"
   },
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "cell_style": "split",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tf.Tensor([0 4 3 2 1 0 0 0], shape=(8,), dtype=int32)\n",
      "tf.Tensor(\n",
      "[[[0.24686503 0.6580968  0.97900116 0.7305386 ]\n",
      "  [0.7398244  0.99561596 0.2881999  0.68756986]\n",
      "  [0.65654933 0.39933133 0.96413505 0.0914396 ]\n",
      "  [0.5710579  0.40055454 0.8896363  0.46409702]]\n",
      "\n",
      " [[0.         0.         0.         0.        ]\n",
      "  [0.         0.         0.         0.        ]\n",
      "  [0.         0.         0.         0.        ]\n",
      "  [0.         0.         0.         0.        ]]\n",
      "\n",
      " [[0.959849   0.03122759 0.6705029  0.32749605]\n",
      "  [0.5580437  0.5194359  0.8763125  0.14576006]\n",
      "  [0.19309628 0.26216495 0.3198707  0.41880834]\n",
      "  [0.8398098  0.8807385  0.41535056 0.6530212 ]]\n",
      "\n",
      " [[0.         0.         0.         0.        ]\n",
      "  [0.         0.         0.         0.        ]\n",
      "  [0.         0.         0.         0.        ]\n",
      "  [0.         0.         0.         0.        ]]], shape=(4, 4, 4), dtype=float32)\n"
     ]
    }
   ],
   "source": [
    "indices = tf.constant([[1],[2],[3],[4]])\n",
    "updates = tf.constant([4,3,2,1])\n",
    "shape = tf.constant([8])\n",
    "\n",
    "print(tf.scatter_nd(indices, updates, shape))\n",
    "\n",
    "indices = tf.constant([[0],[2]])\n",
    "updates = tf.random.uniform([2,4,4])\n",
    "shape = tf.constant([4,4,4])\n",
    "\n",
    "print(tf.scatter_nd(indices, updates, shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "cell_style": "split"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[0.8665, 0.8029, 1.3531],\n",
      "        [0.7052, 0.8495, 1.0810],\n",
      "        [0.8507, 0.6912, 1.3413]])\n",
      "tensor([[0.8665, 0.8029, 1.3531],\n",
      "        [0.7052, 0.8495, 1.0810],\n",
      "        [0.8507, 0.6912, 1.3413]])\n",
      "torch.Size([2, 256])\n"
     ]
    }
   ],
   "source": [
    "a = torch.rand(3,4)\n",
    "b = torch.rand(4,3)\n",
    "\n",
    "print(a@b)\n",
    "print(torch.matmul(a,b))\n",
    "\n",
    "# Example: image x [2, 784] -> [2, 256]\n",
    "# w: [out, in],  w.t() transpose \n",
    "# b: [out]\n",
    "\n",
    "x = torch.rand(2, 784)\n",
    "w = torch.rand(256, 784)\n",
    "b = torch.zeros(256)\n",
    "\n",
    "out = x@w.t() + b\n",
    "print(out.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.4 64-bit ('base': conda)",
   "language": "python",
   "name": "python37464bitbaseconda46a38bc10b2f4a48a71d1cc22a9c8986"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
